{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b226e6af-3d82-4aea-ba8a-e95a2c16a1ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "movieId,title,genres\n",
      "1,Toy Story (1995),Adventure|Animation|Children|Comedy|Fantasy\n",
      "2,Jumanji (1995),Adventure|Children|Fantasy\n",
      "3,Grumpier Old Men (1995),Comedy|Romance\n",
      "4,Waiting to Exhale (1995),Comedy|Drama|Romance\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(\"Dragon Ball Z the Movie: The World's Strongest (a.k.a. Dragon Ball Z: The Strongest Guy in The World) (Doragon bôru Z: Kono yo de ichiban tsuyoi yatsu) (1990)\", 158)\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# the following scripts is for starting a spark program\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q0') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "# this line is to omit unnecessary info\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# read the file in plain text to RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/movies.csv')\n",
    "\n",
    "# print the first 5 lines of the RDD\n",
    "for line in rdd.take(5):\n",
    "    print(line)\n",
    "\n",
    "# the first line of the file contains the header \"movieId,title,genres\", we should ignore this line in our code.\n",
    "header = rdd.first()\n",
    "rdd = rdd.filter(lambda x:x != header)\n",
    "\n",
    "# each line of rdd contains 3 attributes separated by a comma\n",
    "# the title would be the second object of split(',')\n",
    "# we map the line to a tuple (title,len_of_title)\n",
    "rdd = rdd.map(lambda x:(x.split(',')[1],len(x.split(',')[1])))\n",
    "\n",
    "# after the mapping,each line of rdd would be converted to a tuple which contains movie title and its length\n",
    "# a reduce with a simple comparison to get the movie with the longest title\n",
    "result = rdd.reduce(lambda y,x:x if x[1]>y[1] else y)\n",
    "\n",
    "print(result)\n",
    "\n",
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8df911f7-faa7-4879-9eae-5b824860d9d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('65364', 3.6328125)\n",
      "('65533', 3.5259515570934257)\n",
      "('65602', 3.5416666666666665)\n",
      "('65659', 2.6544117647058822)\n",
      "('65752', 5.0)\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q1') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "# Parse and filter data\n",
    "def parse_and_filter(line):\n",
    "    # Skip the header row\n",
    "    if line.startswith(\"userId\"):\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    userId = fields[0]\n",
    "    movieId = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    timestamp = int(fields[3])\n",
    "    \n",
    "    # Check if the rating is within the year 2018\n",
    "    if 1514764800 <= timestamp < 1546300800:  # Timestamp range for the year 2018\n",
    "        return (userId, (rating, 1))\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# Map data and filter out records not in 2018\n",
    "mapped_rdd = rdd.map(parse_and_filter).filter(lambda x: x is not None)\n",
    "\n",
    "# Aggregate data\n",
    "# Use reduceByKey to calculate the total rating and count for each user\n",
    "reduced_rdd = mapped_rdd.reduceByKey(lambda a, b: (a[0] + b[0], a[1] + b[1]))\n",
    "\n",
    "# Calculate average rating\n",
    "average_rating_rdd = reduced_rdd.mapValues(lambda v: v[0] / v[1])\n",
    "\n",
    "# Print the first 5 results\n",
    "for result in average_rating_rdd.take(5):\n",
    "    print(result)\n",
    "\n",
    "# Stop SparkSession\n",
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a26deed7-03c7-4334-b948-fa50fe976a49",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average rating for Romance movies: 3.53\n",
      "Average rating for Comedy movies: 3.41\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q2') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV files from HDFS into RDDs\n",
    "rdd_rating = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "rdd_movies = sc.textFile('hdfs://namenode:9000/input_files/movies.csv')\n",
    "\n",
    "# Step 1: Extract movie IDs for 'Comedy' and 'Romance' genres from movies.csv\n",
    "def parse_movie(line):\n",
    "    if line.startswith(\"movieId\"):  # Skip header row\n",
    "        return []\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    movie_id = fields[0]\n",
    "    title = fields[1]\n",
    "    genres = fields[2].split('|')\n",
    "    \n",
    "    results = []\n",
    "    if 'Comedy' in genres:\n",
    "        results.append(('Comedy', movie_id))\n",
    "    if 'Romance' in genres:\n",
    "        results.append(('Romance', movie_id))\n",
    "    \n",
    "    return results\n",
    "\n",
    "# Filter out header and map movies to their respective genres\n",
    "genre_movie_ids = rdd_movies.flatMap(parse_movie)\n",
    "\n",
    "# Collect all Comedy and Romance movie IDs into a dictionary for quick lookup\n",
    "comedy_movie_ids = set(genre_movie_ids.filter(lambda x: x[0] == 'Comedy').map(lambda x: x[1]).collect())\n",
    "romance_movie_ids = set(genre_movie_ids.filter(lambda x: x[0] == 'Romance').map(lambda x: x[1]).collect())\n",
    "\n",
    "# Step 2: Filter ratings for these movies and calculate average ratings\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (movie_id, rating)\n",
    "\n",
    "# Filter ratings for Comedy and Romance movies\n",
    "ratings_rdd = rdd_rating.map(parse_rating).filter(lambda x: x is not None)\n",
    "\n",
    "# Map ratings to their respective genres\n",
    "def map_ratings_to_genres(rating_tuple):\n",
    "    movie_id, rating = rating_tuple\n",
    "    results = []\n",
    "    if movie_id in comedy_movie_ids:\n",
    "        results.append(('Comedy', (rating, 1)))\n",
    "    if movie_id in romance_movie_ids:\n",
    "        results.append(('Romance', (rating, 1)))\n",
    "    return results\n",
    "\n",
    "mapped_ratings_rdd = ratings_rdd.flatMap(map_ratings_to_genres)\n",
    "\n",
    "# Aggregate data using reduceByKey\n",
    "reduced_ratings_rdd = mapped_ratings_rdd.reduceByKey(lambda a, b: (a[0] + b[0], a[1] + b[1]))\n",
    "\n",
    "# Calculate average ratings\n",
    "average_ratings_rdd = reduced_ratings_rdd.mapValues(lambda v: v[0] / v[1])\n",
    "\n",
    "# Collect and print the results\n",
    "results = average_ratings_rdd.collect()\n",
    "for result in results:\n",
    "    print(f\"Average rating for {result[0]} movies: {result[1]:.2f}\")\n",
    "\n",
    "# Stop SparkSession\n",
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dc232405-f730-47e2-858a-d4598f79763a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Similarity score between user 20 and user 30: 0.0460\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from math import sqrt\n",
    "\n",
    "# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q3') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "user_A = \"20\"\n",
    "user_B = \"30\"\n",
    "\n",
    "# Part 1: Filter records for user A and user B\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (user_id, (movie_id, rating))\n",
    "\n",
    "ratings_rdd = rdd.map(parse_rating).filter(lambda x: x is not None)\n",
    "\n",
    "# Filter ratings for user A and user B\n",
    "ratings_user_A = ratings_rdd.filter(lambda x: x[0] == user_A).map(lambda x: (x[1][0], x[1][1]))\n",
    "ratings_user_B = ratings_rdd.filter(lambda x: x[0] == user_B).map(lambda x: (x[1][0], x[1][1]))\n",
    "\n",
    "# Join the ratings of both users on movie ID\n",
    "common_ratings = ratings_user_A.join(ratings_user_B)\n",
    "\n",
    "# Part 2: Compute the denominator of the formula\n",
    "# Calculate Euclidean norm for each user\n",
    "def compute_euclidean_norm(ratings):\n",
    "    sum_squares = sum(rating ** 2 for rating in ratings)\n",
    "    return sqrt(sum_squares)\n",
    "\n",
    "norm_A = compute_euclidean_norm(ratings_user_A.map(lambda x: x[1]).collect())\n",
    "norm_B = compute_euclidean_norm(ratings_user_B.map(lambda x: x[1]).collect())\n",
    "\n",
    "denominator = norm_A * norm_B\n",
    "\n",
    "# Part 3: Compute the numerator of the formula\n",
    "# Sum of product of ratings for common movies\n",
    "numerator = common_ratings.map(lambda x: x[1][0] * x[1][1]).sum()\n",
    "\n",
    "# Part 4: Compute the similarity score\n",
    "similarity_score = numerator / denominator if denominator != 0 else 0\n",
    "\n",
    "print(f\"Similarity score between user {user_A} and user {user_B}: {similarity_score:.4f}\")\n",
    "\n",
    "# Stop SparkSession\n",
    "spark.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8804d27e",
   "metadata": {},
   "source": [
    "### Q4. Devise a solution for effectively calculating the similarity score matrix, which encompasses all similarity scores for every pair of users. (20%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "887ca56a-6e77-48fb-bda1-37a2a6d7da95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q4') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "# Parse ratings\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (user_id, (movie_id, rating))\n",
    "\n",
    "ratings_rdd = rdd.map(parse_rating).filter(lambda x: x is not None)# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q4') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "# Parse ratings\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (user_id, (movie_id, rating))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "859768de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q4') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "# Parse ratings\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (user_id, (movie_id, rating))\n",
    "\n",
    "ratings_rdd = rdd.map(parse_rating).filter(lambda x: x is not None)# Create SparkSession\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Ass1_Q4') \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Set log level to WARN to reduce unnecessary output\n",
    "spark.sparkContext.setLogLevel('WARN')\n",
    "sc = spark.sparkContext\n",
    "\n",
    "# Read CSV file from HDFS into RDD\n",
    "rdd = sc.textFile('hdfs://namenode:9000/input_files/ratings.csv')\n",
    "\n",
    "# Parse ratings\n",
    "def parse_rating(line):\n",
    "    if line.startswith(\"userId\"):  # Skip header row\n",
    "        return None\n",
    "    \n",
    "    fields = line.split(',')\n",
    "    user_id = fields[0]\n",
    "    movie_id = fields[1]\n",
    "    rating = float(fields[2])\n",
    "    \n",
    "    return (user_id, (movie_id, rating))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
